{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.04586363,  0.04709248,  0.04683661, -0.0475003 ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAApdElEQVR4nO3df3SU5Z338c9MkhkIMJMGSCaRBFEQjBDsAoZZW5eWlIDoSo3PUcsKdjlyZBNPNdZiulbF7jEu7qk/ugh/bFfcPVJae0RXFCyChFoDYkrKL02FQxtcMgnKZoYEmfyY6/nDh3l2FCETQuaayft1zn1O5r6uue/vfZ2E+XDdP8ZhjDECAACwiDPRBQAAAHwRAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWCehAWXVqlW69NJLNWTIEJWUlOi9995LZDkAAMASCQsov/rVr1RVVaVHHnlEf/jDHzR16lSVlZWptbU1USUBAABLOBL1ZYElJSWaMWOG/vVf/1WSFIlEVFBQoHvuuUcPPvhgIkoCAACWSE/ETjs7O1VfX6/q6uroOqfTqdLSUtXV1X2pfzgcVjgcjr6ORCI6ceKERo4cKYfDMSA1AwCAC2OM0cmTJ5Wfny+n89wncRISUD755BP19PQoNzc3Zn1ubq4+/PDDL/WvqanRihUrBqo8AABwER09elRjxow5Z5+EBJR4VVdXq6qqKvo6GAyqsLBQR48elcfjSWBlAACgt0KhkAoKCjRixIjz9k1IQBk1apTS0tLU0tISs76lpUU+n+9L/d1ut9xu95fWezweAgoAAEmmN5dnJOQuHpfLpWnTpmnr1q3RdZFIRFu3bpXf709ESQAAwCIJO8VTVVWlxYsXa/r06brmmmv09NNPq6OjQ9///vcTVRIAALBEwgLKrbfequPHj+vhhx9WIBDQ1Vdfrc2bN3/pwlkAADD4JOw5KBciFArJ6/UqGAxyDQoAAEkins9vvosHAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6/R5QHn30UTkcjphl0qRJ0fbTp0+roqJCI0eO1PDhw1VeXq6Wlpb+LgMAACSxizKDctVVV6m5uTm6vPPOO9G2++67T6+99ppeeukl1dbW6tixY7r55psvRhkAACBJpV+Ujaany+fzfWl9MBjUL37xC61bt07f/va3JUnPP/+8rrzySu3cuVMzZ868GOUAAIAkc1FmUD766CPl5+frsssu08KFC9XU1CRJqq+vV1dXl0pLS6N9J02apMLCQtXV1X3l9sLhsEKhUMwCAABSV78HlJKSEq1du1abN2/W6tWrdeTIEX3zm9/UyZMnFQgE5HK5lJWVFfOe3NxcBQKBr9xmTU2NvF5vdCkoKOjvsgEAgEX6/RTPvHnzoj8XFxerpKREY8eO1a9//WsNHTq0T9usrq5WVVVV9HUoFCKkAACQwi76bcZZWVm64oordOjQIfl8PnV2dqqtrS2mT0tLy1mvWTnD7XbL4/HELAAAIHVd9IDS3t6uw4cPKy8vT9OmTVNGRoa2bt0abW9sbFRTU5P8fv/FLgUAACSJfj/F88Mf/lA33nijxo4dq2PHjumRRx5RWlqabr/9dnm9Xi1ZskRVVVXKzs6Wx+PRPffcI7/fzx08AAAgqt8Dyscff6zbb79dn376qUaPHq1vfOMb2rlzp0aPHi1Jeuqpp+R0OlVeXq5wOKyysjI999xz/V0GAABIYg5jjEl0EfEKhULyer0KBoNcjwIAQJKI5/Ob7+IBAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFgn7oCyY8cO3XjjjcrPz5fD4dArr7wS026M0cMPP6y8vDwNHTpUpaWl+uijj2L6nDhxQgsXLpTH41FWVpaWLFmi9vb2CzoQAACQOuIOKB0dHZo6dapWrVp11vaVK1fq2Wef1Zo1a7Rr1y4NGzZMZWVlOn36dLTPwoULdeDAAW3ZskUbN27Ujh07tHTp0r4fBQAASCkOY4zp85sdDm3YsEELFiyQ9PnsSX5+vu6//3798Ic/lCQFg0Hl5uZq7dq1uu222/TBBx+oqKhIu3fv1vTp0yVJmzdv1vXXX6+PP/5Y+fn5591vKBSS1+tVMBiUx+Ppa/kAAGAAxfP53a/XoBw5ckSBQEClpaXRdV6vVyUlJaqrq5Mk1dXVKSsrKxpOJKm0tFROp1O7du0663bD4bBCoVDMAgAAUle/BpRAICBJys3NjVmfm5sbbQsEAsrJyYlpT09PV3Z2drTPF9XU1Mjr9UaXgoKC/iwbAABYJinu4qmurlYwGIwuR48eTXRJAADgIurXgOLz+SRJLS0tMetbWlqibT6fT62trTHt3d3dOnHiRLTPF7ndbnk8npgFAACkrn4NKOPGjZPP59PWrVuj60KhkHbt2iW/3y9J8vv9amtrU319fbTPtm3bFIlEVFJS0p/lAACAJJUe7xva29t16NCh6OsjR46ooaFB2dnZKiws1L333qt/+qd/0oQJEzRu3Dj95Cc/UX5+fvROnyuvvFJz587VXXfdpTVr1qirq0uVlZW67bbbenUHDwAASH1xB5T3339f3/rWt6Kvq6qqJEmLFy/W2rVr9aMf/UgdHR1aunSp2tra9I1vfEObN2/WkCFDou958cUXVVlZqdmzZ8vpdKq8vFzPPvtsPxwOAABIBRf0HJRE4TkoAAAkn4Q9BwUAAKA/EFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFgn7oCyY8cO3XjjjcrPz5fD4dArr7wS037nnXfK4XDELHPnzo3pc+LECS1cuFAej0dZWVlasmSJ2tvbL+hAAABA6og7oHR0dGjq1KlatWrVV/aZO3eumpubo8svf/nLmPaFCxfqwIED2rJlizZu3KgdO3Zo6dKl8VcPAABSUnq8b5g3b57mzZt3zj5ut1s+n++sbR988IE2b96s3bt3a/r06ZKkn//857r++uv1L//yL8rPz4+3JAAAkGIuyjUo27dvV05OjiZOnKhly5bp008/jbbV1dUpKysrGk4kqbS0VE6nU7t27Trr9sLhsEKhUMwCAABSV78HlLlz5+o//uM/tHXrVv3zP/+zamtrNW/ePPX09EiSAoGAcnJyYt6Tnp6u7OxsBQKBs26zpqZGXq83uhQUFPR32QAAwCJxn+I5n9tuuy3685QpU1RcXKzLL79c27dv1+zZs/u0zerqalVVVUVfh0IhQgoAACnsot9mfNlll2nUqFE6dOiQJMnn86m1tTWmT3d3t06cOPGV16243W55PJ6YBQAApK6LHlA+/vhjffrpp8rLy5Mk+f1+tbW1qb6+Ptpn27ZtikQiKikpudjlAACAJBD3KZ729vbobIgkHTlyRA0NDcrOzlZ2drZWrFih8vJy+Xw+HT58WD/60Y80fvx4lZWVSZKuvPJKzZ07V3fddZfWrFmjrq4uVVZW6rbbbuMOHgAAIElyGGNMPG/Yvn27vvWtb31p/eLFi7V69WotWLBAe/bsUVtbm/Lz8zVnzhz99Kc/VW5ubrTviRMnVFlZqddee01Op1Pl5eV69tlnNXz48F7VEAqF5PV6FQwGOd0DAECSiOfzO+6AYgMCCgAAySeez2++iwcAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArBP3lwUCwECL9HTr0G9Xn7ff+Dl3y5mWMQAVAbjYCCgArNfTFVawad/5OybfV4sB+Aqc4gFgvUjX6USXAGCAEVAAWK+nk4ACDDYEFADWYwYFGHwIKACs101AAQYdAgoA6zGDAgw+BBQA1uvpDCe6BAADjIACwHqRbmZQgMGGgALAem1/Of8zUEbkT5Qc/JMGpAr+mgFY77NPj563z7DRl8rhcAxANQAGAgEFQEpwuoYkugQA/YiAAiAlpKW7JTGDAqQKAgqAlODMcCe6BAD9iIACICWkZbiZQAFSCAEFQEpISx8iEgqQOggoAFICp3iA1EJAAWA1Y0yv+qURUICUQkABYDUT6VFvIorDmcZzUIAUQkABYLVId6fUy1kUAKkjroBSU1OjGTNmaMSIEcrJydGCBQvU2NgY0+f06dOqqKjQyJEjNXz4cJWXl6ulpSWmT1NTk+bPn6/MzEzl5OTogQceUHd394UfDYCUE+kKS72aQwGQSuIKKLW1taqoqNDOnTu1ZcsWdXV1ac6cOero6Ij2ue+++/Taa6/ppZdeUm1trY4dO6abb7452t7T06P58+ers7NT7777rl544QWtXbtWDz/8cP8dFYCU0dMdZgYFGIQcprdXoJ3F8ePHlZOTo9raWl133XUKBoMaPXq01q1bp1tuuUWS9OGHH+rKK69UXV2dZs6cqU2bNumGG27QsWPHlJubK0las2aNli9fruPHj8vlcp13v6FQSF6vV8FgUB6Pp6/lA0gCHZ80qXHjz9QTPnXOfleV/0SZowoGqCoAfRHP5/cFXYMSDAYlSdnZ2ZKk+vp6dXV1qbS0NNpn0qRJKiwsVF1dnSSprq5OU6ZMiYYTSSorK1MoFNKBAwfOup9wOKxQKBSzABgcIl3MoACDUZ8DSiQS0b333qtrr71WkydPliQFAgG5XC5lZWXF9M3NzVUgEIj2+d/h5Ez7mbazqampkdfrjS4FBfwvCRgsIt2diS4BQAL0OaBUVFRo//79Wr9+fX/Wc1bV1dUKBoPR5ejR83/1OoDUEOnu7PWzUACkjvS+vKmyslIbN27Ujh07NGbMmOh6n8+nzs5OtbW1xcyitLS0yOfzRfu89957Mds7c5fPmT5f5Ha75XbzECZgMOo6FZRM5Jx9nBlD5HDy1AQglcT1F22MUWVlpTZs2KBt27Zp3LhxMe3Tpk1TRkaGtm7dGl3X2NiopqYm+f1+SZLf79e+ffvU2toa7bNlyxZ5PB4VFRVdyLEASEH/8+c95z3Nk1U4RelDuWAeSCVxzaBUVFRo3bp1evXVVzVixIjoNSNer1dDhw6V1+vVkiVLVFVVpezsbHk8Ht1zzz3y+/2aOXOmJGnOnDkqKirSHXfcoZUrVyoQCOihhx5SRUUFsyQA+sSRlsFTZIEUE1dAWb16tSRp1qxZMeuff/553XnnnZKkp556Sk6nU+Xl5QqHwyorK9Nzzz0X7ZuWlqaNGzdq2bJl8vv9GjZsmBYvXqzHHnvswo4EwKDlTHdJDk7xAKnkgp6Dkig8BwUYPBpff0qhjz84Z5/c4u/okmk3Ks01ZICqAtAXA/YcFACwweczKJziAVIJAQVA0nOmubiLB0gx/EUDSHrO9Aw5uAYFSCn8RQOwVm8vkeMUD5B6CCgA7GWM1IuM4nCmSSKgAKmEgALAWpGeLplIT6/68hwUILUQUABYy/R0y5znMfcAUhMBBYC14plBAZBaCCgArBVhBgUYtAgoAKxlejqlCAEFGIwIKACsFenp5hQPMEgRUABYi4tkgcGLgALAWpFuLpIFBisCCgBrnQx8pM72E+fskzmqUJkjLxmgigAMFAIKAGuZ7s7zzqCkZQyRM2PIAFUEYKAQUAAkNUdaupzO9ESXAaCfEVAAJDWHM/3/fRcPgFRCQAGQ1BxpaXKkEVCAVENAAZDUnM50OTjFA6QcAgqApOZI4xQPkIoIKACSmsPJKR4gFRFQAFjJGCNjzHn7OZxpcjgIKECqIaAAsJIxPb16iqxDDjkcjgGoCMBAIqAAsJKJRBSJdCe6DAAJQkABYCUT6ZHp6Up0GQAShIACwEom0qNIDzMowGBFQAFgpc9nUAgowGBFQAFgJQIKMLgRUABYiVM8wOBGQAFgpZ7Oz9R9+uQ5+zic6UpzZw5QRQAGEgEFgJXCoeM69UnTOftkDPVoRN6EAaoIwECKK6DU1NRoxowZGjFihHJycrRgwQI1NjbG9Jk1a5YcDkfMcvfdd8f0aWpq0vz585WZmamcnBw98MAD6u5mKhdAnJxOOdMyEl0FgIsgrq8Ara2tVUVFhWbMmKHu7m79+Mc/1pw5c3Tw4EENGzYs2u+uu+7SY489Fn2dmfn/p2B7eno0f/58+Xw+vfvuu2pubtaiRYuUkZGhxx9/vB8OCcBg4XA45Uh3JboMABdBXAFl8+bNMa/Xrl2rnJwc1dfX67rrrouuz8zMlM/nO+s2fvvb3+rgwYN66623lJubq6uvvlo//elPtXz5cj366KNyufjHBkDvOJxOOdOZQQFS0QVdgxIMBiVJ2dnZMetffPFFjRo1SpMnT1Z1dbVOnToVbaurq9OUKVOUm5sbXVdWVqZQKKQDBw6cdT/hcFihUChmAQA5HJziAVJUXDMo/1skEtG9996ra6+9VpMnT46u/973vqexY8cqPz9fe/fu1fLly9XY2KiXX35ZkhQIBGLCiaTo60AgcNZ91dTUaMWKFX0tFUCKcjiYQQFSVZ8DSkVFhfbv36933nknZv3SpUujP0+ZMkV5eXmaPXu2Dh8+rMsvv7xP+6qurlZVVVX0dSgUUkFBQd8KB5AyHE6nHMygACmpT6d4KisrtXHjRr399tsaM2bMOfuWlJRIkg4dOiRJ8vl8amlpielz5vVXXbfidrvl8XhiFgCQg7t4gFQVV0AxxqiyslIbNmzQtm3bNG7cuPO+p6GhQZKUl5cnSfL7/dq3b59aW1ujfbZs2SKPx6OioqJ4ygGQoowxverHKR4gdcV1iqeiokLr1q3Tq6++qhEjRkSvGfF6vRo6dKgOHz6sdevW6frrr9fIkSO1d+9e3XfffbruuutUXFwsSZozZ46Kiop0xx13aOXKlQoEAnrooYdUUVEht9vd/0cIICn17nt4HHI4eN4kkIri+stevXq1gsGgZs2apby8vOjyq1/9SpLkcrn01ltvac6cOZo0aZLuv/9+lZeX67XXXotuIy0tTRs3blRaWpr8fr/+7u/+TosWLYp5bgqAQc5EFOnuTHQVABIorhmU8027FhQUqLa29rzbGTt2rN544414dg1gEDHGqIeAAgxqzI0CsI8xzKAAgxwBBYB1jAgowGBHQAFgH65BAQY9AgoA+3CKBxj0CCgArGOMUaSLgAIMZgQUANbpDnfoxOHd5+zjcKZr1BX+AaoIwEAjoACwjzEykfM8qM3hUPqQYQNTD4ABR0ABkLScGTx9GkhVBBQAScnhcBBQgBRGQAGQtNLSCShAqiKgAEhSDjkJKEDKIqAASE4OKc1FQAFSFQEFQJJiBgVIZQQUAEnJ4ZDSuEgWSFkEFABWMcbI9JznGSiSJIfk4J8wIFXx1w3AOj1d4USXACDBCCgArBPpOp3oEgAkGAEFgHWYQQFAQAFgnR5mUIBBLz3RBQBILZFIRJFIpO8bMEZd4c960c2op6enz7txOBxKS0vr8/sBXFwEFAD96sEHH9RTTz3V5/c7HNLt356syu/OOGe/UCikoUOH9nk/t912m/7zP/+zz+8HcHERUAD0q0gkou7u3twmfHYOh+TOOP/ZZ2PMBe3nQmZfAFx8BBQAVnHIoRmT8iVJ7d1Z+rQrX+HIULmcp/W1jIC86Z9KkhoOBRJZJoCLjIACwCoOhzRlXK5OdOXqYPu1OhXxqMdkKE3dGpoW0hWZ7yvX/RfVNzYnulQAFxF38QCwTkePV38Ilelkzyj1GJckh3qUofaekdrbPkv/05WjU+GuRJcJ4CIioACwipFTv2v7P+oyQ87a3m3c2hm8SW2nHANcGYCBREABYKHzhQ8HMyhAiiOgAEhKp053JroEABcRAQVAUjp1mhkUIJURUABYxaGI/N4NcurszzhxqkfTPa+rs/PUAFcGYCDFFVBWr16t4uJieTweeTwe+f1+bdq0Kdp++vRpVVRUaOTIkRo+fLjKy8vV0tISs42mpibNnz9fmZmZysnJ0QMPPHBBD1sCkHq86Z9oumezMp3B/xdUjJzq1lDnSRWPeFujMv5bp8Kc4gFSWVzPQRkzZoyeeOIJTZgwQcYYvfDCC7rpppu0Z88eXXXVVbrvvvv0+uuv66WXXpLX61VlZaVuvvlm/f73v5f0+ZMb58+fL5/Pp3fffVfNzc1atGiRMjIy9Pjjj1+UAwSQXCLG6NXfN8rp/JNC3X9Ua+dYnY4Mk8vxmUa7jqot4/P/9HRwDQqQ0hzGGHMhG8jOztaTTz6pW265RaNHj9a6det0yy23SJI+/PBDXXnllaqrq9PMmTO1adMm3XDDDTp27Jhyc3MlSWvWrNHy5ct1/PhxuVyuXu0zFArJ6/Xqzjvv7PV7AAyMnTt3au/evYku47zGjx+vb3/724kuAxhUOjs7tXbtWgWDQXk8nnP27fOTZHt6evTSSy+po6NDfr9f9fX16urqUmlpabTPpEmTVFhYGA0odXV1mjJlSjScSFJZWZmWLVumAwcO6Otf//pZ9xUOhxUOh6OvQ6GQJOmOO+7Q8OHD+3oIAC6Cjo6OpAgol112mZYsWZLoMoBBpb29XWvXru1V37gDyr59++T3+3X69GkNHz5cGzZsUFFRkRoaGuRyuZSVlRXTPzc3V4HA59+ZEQgEYsLJmfYzbV+lpqZGK1as+NL66dOnnzeBARhYPp8v0SX0ysiRI3XNNdckugxgUDkzwdAbcd/FM3HiRDU0NGjXrl1atmyZFi9erIMHD8a7mbhUV1crGAxGl6NHj17U/QEAgMSKewbF5XJp/PjxkqRp06Zp9+7deuaZZ3Trrbeqs7NTbW1tMbMoLS0t0f9R+Xw+vffeezHbO3OXz7n+1+V2u+V2u+MtFQAAJKkLfg5KJBJROBzWtGnTlJGRoa1bt0bbGhsb1dTUJL/fL0ny+/3at2+fWltbo322bNkij8ejoqKiCy0FAACkiLhmUKqrqzVv3jwVFhbq5MmTWrdunbZv364333xTXq9XS5YsUVVVlbKzs+XxeHTPPffI7/dr5syZkqQ5c+aoqKhId9xxh1auXKlAIKCHHnpIFRUVzJAAAICouAJKa2urFi1apObmZnm9XhUXF+vNN9/Ud77zHUnSU089JafTqfLycoXDYZWVlem5556Lvj8tLU0bN27UsmXL5Pf7NWzYMC1evFiPPfZY/x4VAABIanEFlF/84hfnbB8yZIhWrVqlVatWfWWfsWPH6o033ohntwAAYJDhu3gAAIB1CCgAAMA6BBQAAGAdAgoAALBOn7+LBwDOZvLkyVqwYEGiyziv6dOnJ7oEAOdwwd9mnAhnvs24N9+GCAAA7BDP5zeneAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOvEFVBWr16t4uJieTweeTwe+f1+bdq0Kdo+a9YsORyOmOXuu++O2UZTU5Pmz5+vzMxM5eTk6IEHHlB3d3f/HA0AAEgJ6fF0HjNmjJ544glNmDBBxhi98MILuummm7Rnzx5dddVVkqS77rpLjz32WPQ9mZmZ0Z97eno0f/58+Xw+vfvuu2pubtaiRYuUkZGhxx9/vJ8OCQAAJDuHMcZcyAays7P15JNPasmSJZo1a5auvvpqPf3002ftu2nTJt1www06duyYcnNzJUlr1qzR8uXLdfz4cblcrl7tMxQKyev1KhgMyuPxXEj5AABggMTz+d3na1B6enq0fv16dXR0yO/3R9e/+OKLGjVqlCZPnqzq6mqdOnUq2lZXV6cpU6ZEw4kklZWVKRQK6cCBA1+5r3A4rFAoFLMAAIDUFdcpHknat2+f/H6/Tp8+reHDh2vDhg0qKiqSJH3ve9/T2LFjlZ+fr71792r58uVqbGzUyy+/LEkKBAIx4URS9HUgEPjKfdbU1GjFihXxlgoAAJJU3AFl4sSJamhoUDAY1G9+8xstXrxYtbW1Kioq0tKlS6P9pkyZory8PM2ePVuHDx/W5Zdf3uciq6urVVVVFX0dCoVUUFDQ5+0BAAC7xX2Kx+Vyafz48Zo2bZpqamo0depUPfPMM2ftW1JSIkk6dOiQJMnn86mlpSWmz5nXPp/vK/fpdrujdw6dWQAAQOq64OegRCIRhcPhs7Y1NDRIkvLy8iRJfr9f+/btU2tra7TPli1b5PF4oqeJAAAA4jrFU11drXnz5qmwsFAnT57UunXrtH37dr355ps6fPiw1q1bp+uvv14jR47U3r17dd999+m6665TcXGxJGnOnDkqKirSHXfcoZUrVyoQCOihhx5SRUWF3G73RTlAAACQfOIKKK2trVq0aJGam5vl9XpVXFysN998U9/5znd09OhRvfXWW3r66afV0dGhgoIClZeX66GHHoq+Py0tTRs3btSyZcvk9/s1bNgwLV68OOa5KQAAABf8HJRE4DkoAAAknwF5DgoAAMDFQkABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKyTnugC+sIYI0kKhUIJrgQAAPTWmc/tM5/j55KUAeXkyZOSpIKCggRXAgAA4nXy5El5vd5z9nGY3sQYy0QiETU2NqqoqEhHjx6Vx+NJdElJKxQKqaCggHHsB4xl/2Es+wfj2H8Yy/5hjNHJkyeVn58vp/PcV5kk5QyK0+nUJZdcIknyeDz8svQDxrH/MJb9h7HsH4xj/2EsL9z5Zk7O4CJZAABgHQIKAACwTtIGFLfbrUceeURutzvRpSQ1xrH/MJb9h7HsH4xj/2EsB15SXiQLAABSW9LOoAAAgNRFQAEAANYhoAAAAOsQUAAAgHWSMqCsWrVKl156qYYMGaKSkhK99957iS7JOjt27NCNN96o/Px8ORwOvfLKKzHtxhg9/PDDysvL09ChQ1VaWqqPPvoops+JEye0cOFCeTweZWVlacmSJWpvbx/Ao0i8mpoazZgxQyNGjFBOTo4WLFigxsbGmD6nT59WRUWFRo4cqeHDh6u8vFwtLS0xfZqamjR//nxlZmYqJydHDzzwgLq7uwfyUBJq9erVKi4ujj7kyu/3a9OmTdF2xrDvnnjiCTkcDt17773RdYxn7zz66KNyOBwxy6RJk6LtjGOCmSSzfv1643K5zL//+7+bAwcOmLvuustkZWWZlpaWRJdmlTfeeMP84z/+o3n55ZeNJLNhw4aY9ieeeMJ4vV7zyiuvmD/+8Y/mb//2b824cePMZ599Fu0zd+5cM3XqVLNz507zu9/9zowfP97cfvvtA3wkiVVWVmaef/55s3//ftPQ0GCuv/56U1hYaNrb26N97r77blNQUGC2bt1q3n//fTNz5kzz13/919H27u5uM3nyZFNaWmr27Nlj3njjDTNq1ChTXV2diENKiP/6r/8yr7/+uvnTn/5kGhsbzY9//GOTkZFh9u/fb4xhDPvqvffeM5deeqkpLi42P/jBD6LrGc/eeeSRR8xVV11lmpubo8vx48ej7YxjYiVdQLnmmmtMRUVF9HVPT4/Jz883NTU1CazKbl8MKJFIxPh8PvPkk09G17W1tRm3221++ctfGmOMOXjwoJFkdu/eHe2zadMm43A4zH//938PWO22aW1tNZJMbW2tMebzccvIyDAvvfRStM8HH3xgJJm6ujpjzOdh0el0mkAgEO2zevVq4/F4TDgcHtgDsMjXvvY182//9m+MYR+dPHnSTJgwwWzZssX8zd/8TTSgMJ6998gjj5ipU6eetY1xTLykOsXT2dmp+vp6lZaWRtc5nU6Vlpaqrq4ugZUllyNHjigQCMSMo9frVUlJSXQc6+rqlJWVpenTp0f7lJaWyul0ateuXQNesy2CwaAkKTs7W5JUX1+vrq6umLGcNGmSCgsLY8ZyypQpys3NjfYpKytTKBTSgQMHBrB6O/T09Gj9+vXq6OiQ3+9nDPuooqJC8+fPjxk3id/JeH300UfKz8/XZZddpoULF6qpqUkS42iDpPqywE8++UQ9PT0xvwySlJubqw8//DBBVSWfQCAgSWcdxzNtgUBAOTk5Me3p6enKzs6O9hlsIpGI7r33Xl177bWaPHmypM/HyeVyKSsrK6bvF8fybGN9pm2w2Ldvn/x+v06fPq3hw4drw4YNKioqUkNDA2MYp/Xr1+sPf/iDdu/e/aU2fid7r6SkRGvXrtXEiRPV3NysFStW6Jvf/Kb279/POFogqQIKkEgVFRXav3+/3nnnnUSXkpQmTpyohoYGBYNB/eY3v9HixYtVW1ub6LKSztGjR/WDH/xAW7Zs0ZAhQxJdTlKbN29e9Ofi4mKVlJRo7Nix+vWvf62hQ4cmsDJISXYXz6hRo5SWlvalq6hbWlrk8/kSVFXyOTNW5xpHn8+n1tbWmPbu7m6dOHFiUI51ZWWlNm7cqLfffltjxoyJrvf5fOrs7FRbW1tM/y+O5dnG+kzbYOFyuTR+/HhNmzZNNTU1mjp1qp555hnGME719fVqbW3VX/3VXyk9PV3p6emqra3Vs88+q/T0dOXm5jKefZSVlaUrrrhChw4d4vfSAkkVUFwul6ZNm6atW7dG10UiEW3dulV+vz+BlSWXcePGyefzxYxjKBTSrl27ouPo9/vV1tam+vr6aJ9t27YpEomopKRkwGtOFGOMKisrtWHDBm3btk3jxo2LaZ82bZoyMjJixrKxsVFNTU0xY7lv376YwLdlyxZ5PB4VFRUNzIFYKBKJKBwOM4Zxmj17tvbt26eGhoboMn36dC1cuDD6M+PZN+3t7Tp8+LDy8vL4vbRBoq/Sjdf69euN2+02a9euNQcPHjRLly41WVlZMVdR4/Mr/Pfs2WP27NljJJmf/exnZs+ePeYvf/mLMebz24yzsrLMq6++avbu3Wtuuumms95m/PWvf93s2rXLvPPOO2bChAmD7jbjZcuWGa/Xa7Zv3x5zK+KpU6eife6++25TWFhotm3bZt5//33j9/uN3++Ptp+5FXHOnDmmoaHBbN682YwePXpQ3Yr44IMPmtraWnPkyBGzd+9e8+CDDxqHw2F++9vfGmMYwwv1v+/iMYbx7K3777/fbN++3Rw5csT8/ve/N6WlpWbUqFGmtbXVGMM4JlrSBRRjjPn5z39uCgsLjcvlMtdcc43ZuXNnokuyzttvv20kfWlZvHixMebzW41/8pOfmNzcXON2u83s2bNNY2NjzDY+/fRTc/vtt5vhw4cbj8djvv/975uTJ08m4GgS52xjKMk8//zz0T6fffaZ+Yd/+Afzta99zWRmZprvfve7prm5OWY7f/7zn828efPM0KFDzahRo8z9999vurq6BvhoEufv//7vzdixY43L5TKjR482s2fPjoYTYxjDC/XFgMJ49s6tt95q8vLyjMvlMpdccom59dZbzaFDh6LtjGNiOYwxJjFzNwAAAGeXVNegAACAwYGAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADr/F/Y9Q59zZ89eQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2549/1318187111.py:141: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  states = torch.FloatTensor(states).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0, 13.0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import random\n",
    "from IPython import display\n",
    "\n",
    "\n",
    "class PPO:\n",
    "    def __init__(self):\n",
    "        #定义模型\n",
    "        self.model = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 2),\n",
    "            torch.nn.Softmax(dim=1),\n",
    "        )\n",
    "\n",
    "        self.model_td = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "        )\n",
    "\n",
    "        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=1e-3)\n",
    "        self.optimizer_td = torch.optim.Adam(self.model_td.parameters(),\n",
    "                                             lr=1e-2)\n",
    "        self.loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #得到一个动作\n",
    "    def get_action(self, state):\n",
    "        state = torch.FloatTensor(state).reshape(1, 4)\n",
    "        #[1, 4] -> [1, 2]\n",
    "        prob = self.model(state)\n",
    "\n",
    "        #根据概率选择一个动作\n",
    "        action = random.choices(range(2), weights=prob[0].tolist(), k=1)[0]\n",
    "\n",
    "        return action\n",
    "\n",
    "    def _get_advantages(self, deltas):\n",
    "        advantages = []\n",
    "\n",
    "        #反向遍历deltas\n",
    "        s = 0.0\n",
    "        for delta in deltas[::-1]:\n",
    "            s = 0.98 * 0.95 * s + delta\n",
    "            advantages.append(s)\n",
    "\n",
    "        #逆序\n",
    "        advantages.reverse()\n",
    "        return advantages\n",
    "\n",
    "    def train(self, states, rewards, actions, next_states, overs):\n",
    "        #states -> [b, 4]\n",
    "        #rewards -> [b, 1]\n",
    "        #actions -> [b, 1]\n",
    "        #next_states -> [b, 4]\n",
    "        #overs -> [b, 1]\n",
    "\n",
    "        #计算values和targets\n",
    "        #[b, 4] -> [b, 1]\n",
    "        values = self.model_td(states)\n",
    "\n",
    "        #[b, 4] -> [b, 1]\n",
    "        targets = self.model_td(next_states) * 0.98\n",
    "        targets *= (1 - overs)\n",
    "        targets += rewards\n",
    "\n",
    "        #[b, 1]\n",
    "        deltas = (targets - values).squeeze(dim=1).tolist()\n",
    "        advantages = self._get_advantages(deltas)\n",
    "        advantages = torch.FloatTensor(advantages).reshape(-1, 1)\n",
    "\n",
    "        #取出每一步的动作概率\n",
    "        #[b, 2] -> [b, 2] -> [b, 1]\n",
    "        old_probs = self.model(states)\n",
    "        old_probs = old_probs.gather(1, actions)\n",
    "        old_probs = old_probs.detach()\n",
    "\n",
    "        #每批数据反复训练10次\n",
    "        for _ in range(10):\n",
    "            #[b, 4] -> [b, 2]\n",
    "            new_probs = self.model(states)\n",
    "\n",
    "            #[b, 2] -> [b, 1]\n",
    "            new_probs = new_probs.gather(1, actions)\n",
    "            new_probs = new_probs\n",
    "\n",
    "            #[b, 1] - [b, 1] -> [b, 1]\n",
    "            ratios = new_probs / old_probs\n",
    "\n",
    "            #[b, 1] * [b, 1] -> [b, 1]\n",
    "            surr1 = ratios * advantages\n",
    "\n",
    "            #[b, 1] * [b, 1] -> [b, 1]\n",
    "            surr2 = torch.clamp(ratios, 0.8, 1.2) * advantages\n",
    "\n",
    "            loss = -torch.min(surr1, surr2)\n",
    "            loss = loss.mean()\n",
    "\n",
    "            values = self.model_td(states)\n",
    "            loss_td = self.loss_fn(values, targets.detach())\n",
    "\n",
    "            #更新参数\n",
    "            self.optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            self.optimizer.step()\n",
    "\n",
    "            self.optimizer_td.zero_grad()\n",
    "            loss_td.backward()\n",
    "            self.optimizer_td.step()\n",
    "\n",
    "    def get_data(self):\n",
    "        states = []\n",
    "        rewards = []\n",
    "        actions = []\n",
    "        next_states = []\n",
    "        overs = []\n",
    "\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action = self.get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            next_state, reward, over, _ = env.step(action)\n",
    "\n",
    "            #记录数据样本\n",
    "            states.append(state)\n",
    "            rewards.append(reward)\n",
    "            actions.append(action)\n",
    "            next_states.append(next_state)\n",
    "            overs.append(over)\n",
    "\n",
    "            #更新游戏状态,开始下一个动作\n",
    "            state = next_state\n",
    "\n",
    "        #[b, 4]\n",
    "        states = torch.FloatTensor(states).reshape(-1, 4)\n",
    "        #[b, 1]\n",
    "        rewards = torch.FloatTensor(rewards).reshape(-1, 1)\n",
    "        #[b, 1]\n",
    "        actions = torch.LongTensor(actions).reshape(-1, 1)\n",
    "        #[b, 4]\n",
    "        next_states = torch.FloatTensor(next_states).reshape(-1, 4)\n",
    "        #[b, 1]\n",
    "        overs = torch.LongTensor(overs).reshape(-1, 1)\n",
    "\n",
    "        return states, rewards, actions, next_states, overs\n",
    "\n",
    "    def test(self, play):\n",
    "        #初始化游戏\n",
    "        state = env.reset()\n",
    "\n",
    "        #记录反馈值的和,这个值越大越好\n",
    "        reward_sum = 0\n",
    "\n",
    "        #玩到游戏结束为止\n",
    "        over = False\n",
    "        while not over:\n",
    "            #根据当前状态得到一个动作\n",
    "            action = self.get_action(state)\n",
    "\n",
    "            #执行动作,得到反馈\n",
    "            state, reward, over, _ = env.step(action)\n",
    "            reward_sum += reward\n",
    "\n",
    "            #打印动画\n",
    "            if play and random.random() < 0.2:  #跳帧\n",
    "                display.clear_output(wait=True)\n",
    "                show()\n",
    "\n",
    "        return reward_sum\n",
    "\n",
    "\n",
    "teacher = PPO()\n",
    "\n",
    "teacher.train(*teacher.get_data())\n",
    "\n",
    "teacher.get_action([1, 2, 3, 4]), teacher.test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 17.0\n",
      "50 174.1\n",
      "100 187.7\n",
      "150 200.0\n",
      "200 200.0\n",
      "250 200.0\n",
      "300 200.0\n",
      "350 200.0\n",
      "400 200.0\n",
      "450 200.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(500):\n",
    "    teacher.train(*teacher.get_data())\n",
    "\n",
    "    if i % 50 == 0:\n",
    "        test_result = sum([teacher.test(play=False) for _ in range(10)]) / 10\n",
    "        print(i, test_result)\n",
    "\n",
    "teacher.test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([200, 4]), torch.Size([200, 1]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#使用训练好的模型获取一批教师数据\n",
    "teacher_states, _, teacher_actions, _, _ = teacher.get_data()\n",
    "\n",
    "#删除教师,只留下教师的数据就可以了\n",
    "del teacher\n",
    "\n",
    "teacher_states.shape, teacher_actions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.PPO at 0x7fa7d8536fa0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#初始化学生模型\n",
    "student = PPO()\n",
    "\n",
    "student"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4961],\n",
       "        [0.5672]], grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#定义鉴别器网络,它的任务是鉴定一批数据是出自teacher还是student\n",
    "class Discriminator(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.sequential = torch.nn.Sequential(\n",
    "            torch.nn.Linear(6, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "            torch.nn.Sigmoid(),\n",
    "        )\n",
    "\n",
    "    def forward(self, states, actions):\n",
    "        one_hot = torch.nn.functional.one_hot(actions.squeeze(dim=1),\n",
    "                                              num_classes=2)\n",
    "\n",
    "        cat = torch.cat([states, one_hot], dim=1)\n",
    "\n",
    "        return self.sequential(cat)\n",
    "\n",
    "\n",
    "discriminator = Discriminator()\n",
    "\n",
    "discriminator(torch.randn(2, 4), torch.ones(2, 1).long())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 14.9\n",
      "50 156.6\n",
      "100 200.0\n",
      "150 200.0\n",
      "200 200.0\n",
      "250 200.0\n",
      "300 200.0\n",
      "350 200.0\n",
      "400 200.0\n",
      "450 200.0\n"
     ]
    }
   ],
   "source": [
    "#模仿学习\n",
    "def copy_learn():\n",
    "    optimizer = torch.optim.Adam(discriminator.parameters(), lr=1e-3)\n",
    "    bce_loss = torch.nn.BCELoss()\n",
    "\n",
    "    for i in range(500):\n",
    "        #使用学生模型获取一局游戏的数据,不需要reward\n",
    "        states, _, actions, next_states, overs = student.get_data()\n",
    "\n",
    "        #使用鉴别器鉴定两批数据是来自教师的还是学生的\n",
    "        prob_teacher = discriminator(teacher_states, teacher_actions)\n",
    "        prob_student = discriminator(states, actions)\n",
    "\n",
    "        #老师的用0表示,学生的用1表示,计算二分类loss\n",
    "        loss_teacher = bce_loss(prob_teacher, torch.zeros_like(prob_teacher))\n",
    "        loss_student = bce_loss(prob_student, torch.ones_like(prob_student))\n",
    "        loss = loss_teacher + loss_student\n",
    "\n",
    "        #调整鉴别器的loss\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        #使用一批数据来自学生的概率作为reward,取log,再符号取反\n",
    "        #因为鉴别器会把学生数据的概率贴近1,所以目标是让鉴别器无法分辨,这是一种对抗网络的思路\n",
    "        rewards = -prob_student.log().detach()\n",
    "\n",
    "        #更新学生模型参数,使用PPO模型本身的更新方式\n",
    "        student.train(states, rewards, actions, next_states, overs)\n",
    "\n",
    "        if i % 50 == 0:\n",
    "            test_result = sum([student.test(play=False)\n",
    "                               for _ in range(10)]) / 10\n",
    "            print(i, test_result)\n",
    "\n",
    "\n",
    "copy_learn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAApNUlEQVR4nO3df3BUVZ738U93kg6E0J0JkHQiCaIwQITgLGjodXSYJRIQXRljlTosxFlKSjaxRuM4mFlGxNkyrm6tP2YU/thdceuRwcESXRmBQZAwjgExkiGAZoRiJ7ikE5RNN4nmV/d5/uDhPtOKSHfS9O34flXdqvQ9p7u/9xTV/eHcc287jDFGAAAANuJMdAEAAABfREABAAC2Q0ABAAC2Q0ABAAC2Q0ABAAC2Q0ABAAC2Q0ABAAC2Q0ABAAC2Q0ABAAC2Q0ABAAC2k9CA8uyzz+rSSy/VsGHDVFJSonfffTeR5QAAAJtIWEB56aWXVF1drVWrVun999/X9OnTVVZWpvb29kSVBAAAbMKRqB8LLCkp0VVXXaVf/epXkqRwOKyCggLdc889evDBBxNREgAAsInURLxpb2+vGhoaVFNTY+1zOp0qLS1VfX39l/r39PSop6fHehwOh3Xq1CmNGjVKDofjotQMAAAGxhij06dPKz8/X07n+U/iJCSgfPLJJwqFQsrNzY3Yn5ubqw8//PBL/Wtra7V69eqLVR4AAIij48ePa+zYseftk5CAEq2amhpVV1dbjwOBgAoLC3X8+HG53e4EVgYAAC5UMBhUQUGBRo4c+bV9ExJQRo8erZSUFLW1tUXsb2trk9fr/VL/9PR0paenf2m/2+0moAAAkGQuZHlGQq7icblcmjFjhnbs2GHtC4fD2rFjh3w+XyJKAgAANpKwUzzV1dWqqKjQzJkzdfXVV+upp55SV1eXfvSjHyWqJAAAYBMJCyi33XabTp48qYceekh+v19XXnmltm7d+qWFswAA4JsnYfdBGYhgMCiPx6NAIMAaFAAAkkQ039/8Fg8AALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALAdAgoAALCdQQ8oDz/8sBwOR8Q2efJkq727u1uVlZUaNWqUMjMzVV5erra2tsEuAwAAJLG4zKBcccUVam1ttba3337barvvvvv0+uuva+PGjaqrq9OJEyd0yy23xKMMAACQpFLj8qKpqfJ6vV/aHwgE9O///u9av369/uZv/kaS9Pzzz2vKlCnas2ePZs2aFY9yAABAkonLDMpHH32k/Px8XXbZZVq0aJFaWlokSQ0NDerr61NpaanVd/LkySosLFR9ff1Xvl5PT4+CwWDEBgAAhq5BDyglJSVat26dtm7dqjVr1ujYsWO69tprdfr0afn9frlcLmVlZUU8Jzc3V36//ytfs7a2Vh6Px9oKCgoGu2wAAGAjg36KZ/78+dbfxcXFKikp0bhx4/Sb3/xGw4cPj+k1a2pqVF1dbT0OBoOEFAAAhrC4X2aclZWlb3/72zpy5Ii8Xq96e3vV0dER0aetre2ca1bOSk9Pl9vtjtgAAMDQFfeA0tnZqaNHjyovL08zZsxQWlqaduzYYbU3NzerpaVFPp8v3qUAAIAkMeineH7yk5/opptu0rhx43TixAmtWrVKKSkpuuOOO+TxeLR06VJVV1crOztbbrdb99xzj3w+H1fwAAAAy6AHlI8//lh33HGHPv30U40ZM0bf/e53tWfPHo0ZM0aS9OSTT8rpdKq8vFw9PT0qKyvTc889N9hlAACAJOYwxphEFxGtYDAoj8ejQCDAehQAAJJENN/f/BYPAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwHQIKAACwnagDyu7du3XTTTcpPz9fDodDr776akS7MUYPPfSQ8vLyNHz4cJWWluqjjz6K6HPq1CktWrRIbrdbWVlZWrp0qTo7Owd0IAAAYOiIOqB0dXVp+vTpevbZZ8/Z/vjjj+uZZ57R2rVrtXfvXo0YMUJlZWXq7u62+ixatEiHDh3S9u3btXnzZu3evVvLli2L/SgAAMCQ4jDGmJif7HBo06ZNWrhwoaQzsyf5+fm6//779ZOf/ESSFAgElJubq3Xr1un222/XBx98oKKiIu3bt08zZ86UJG3dulU33HCDPv74Y+Xn53/t+waDQXk8HgUCAbnd7ljLBwAAF1E039+Dugbl2LFj8vv9Ki0ttfZ5PB6VlJSovr5eklRfX6+srCwrnEhSaWmpnE6n9u7de87X7enpUTAYjNgAAMDQNagBxe/3S5Jyc3Mj9ufm5lptfr9fOTk5Ee2pqanKzs62+nxRbW2tPB6PtRUUFAxm2QAAwGaS4iqempoaBQIBazt+/HiiSwIAAHE0qAHF6/VKktra2iL2t7W1WW1er1ft7e0R7f39/Tp16pTV54vS09PldrsjNgAAMHQNakAZP368vF6vduzYYe0LBoPau3evfD6fJMnn86mjo0MNDQ1Wn507dyocDqukpGQwywEAAEkqNdondHZ26siRI9bjY8eOqbGxUdnZ2SosLNS9996rf/qnf9LEiRM1fvx4/fznP1d+fr51pc+UKVM0b9483XXXXVq7dq36+vpUVVWl22+//YKu4AEAAENf1AHlvffe0/e//33rcXV1tSSpoqJC69at009/+lN1dXVp2bJl6ujo0He/+11t3bpVw4YNs57z4osvqqqqSnPmzJHT6VR5ebmeeeaZQTgcAAAwFAzoPiiJwn1QAABIPgm7DwoAAMBgIKAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbIaAAAADbiTqg7N69WzfddJPy8/PlcDj06quvRrTfeeedcjgcEdu8efMi+pw6dUqLFi2S2+1WVlaWli5dqs7OzgEdCAAAGDqiDihdXV2aPn26nn322a/sM2/ePLW2tlrbr3/964j2RYsW6dChQ9q+fbs2b96s3bt3a9myZdFXDwAAhqTUaJ8wf/58zZ8//7x90tPT5fV6z9n2wQcfaOvWrdq3b59mzpwpSfrlL3+pG264Qf/yL/+i/Pz8aEsCAABDTFzWoOzatUs5OTmaNGmSli9frk8//dRqq6+vV1ZWlhVOJKm0tFROp1N79+495+v19PQoGAxGbAAAYOga9IAyb948/ed//qd27Nihf/7nf1ZdXZ3mz5+vUCgkSfL7/crJyYl4TmpqqrKzs+X3+8/5mrW1tfJ4PNZWUFAw2GUDAAAbifoUz9e5/fbbrb+nTZum4uJiXX755dq1a5fmzJkT02vW1NSourraehwMBgkpAAAMYXG/zPiyyy7T6NGjdeTIEUmS1+tVe3t7RJ/+/n6dOnXqK9etpKeny+12R2wAAGDointA+fjjj/Xpp58qLy9PkuTz+dTR0aGGhgarz86dOxUOh1VSUhLvcgAAQBKI+hRPZ2enNRsiSceOHVNjY6Oys7OVnZ2t1atXq7y8XF6vV0ePHtVPf/pTTZgwQWVlZZKkKVOmaN68ebrrrru0du1a9fX1qaqqSrfffjtX8AAAAEmSwxhjonnCrl279P3vf/9L+ysqKrRmzRotXLhQ+/fvV0dHh/Lz8zV37lz94he/UG5urtX31KlTqqqq0uuvvy6n06ny8nI988wzyszMvKAagsGgPB6PAoEAp3sAAEgS0Xx/Rx1Q7ICAAgBA8onm+5vf4gEAALZDQAEAALZDQAEAALZDQAEAALZDQAEAALZDQAEAALZDQAEAALZDQAEAALZDQAEAALZDQAEAALYT9Y8FAkDf56d1bNe68/ZxpqTq8uvvlsPhuDhFARhSCCgAomZCfQq0NJ23jzPVJRMOyZHCxwyA6HGKB0DchPv7El0CgCRFQAEQF8YYmVB/ossAkKQIKADiJhzqTXQJAJIUAQVA3DCDAiBWBBQAcRMOsQYFQGwIKADixBBQAMSMgAIgbgwBBUCMCCgA4sNwmTGA2BFQAMRNmEWyAGJEQAEQJ4ZTPABiRkABEDfMoACIFQEFQNQczlQN8+Set48xYXV98ueLVBGAoYaAAiBqztQ0jci9/PydjFHniT9dnIIADDkEFAAxcMjJrxQDiCMCCoDoORxyEFAAxBEBBUBMmEEBEE8EFABRczgccqSkJboMAEMYAQVADBxyOplBARA/UQWU2tpaXXXVVRo5cqRycnK0cOFCNTc3R/Tp7u5WZWWlRo0apczMTJWXl6utrS2iT0tLixYsWKCMjAzl5OTogQceUH8/90sAkoZDrEEBEFdRBZS6ujpVVlZqz5492r59u/r6+jR37lx1dXVZfe677z69/vrr2rhxo+rq6nTixAndcsstVnsoFNKCBQvU29urd955Ry+88ILWrVunhx56aPCOCkCcsUgWQHw5jDEm1iefPHlSOTk5qqur03XXXadAIKAxY8Zo/fr1uvXWWyVJH374oaZMmaL6+nrNmjVLW7Zs0Y033qgTJ04oN/fMjZ7Wrl2rFStW6OTJk3K5XF/7vsFgUB6PR4FAQG63O9byAcTIhEM6+eEf9Off/5/z9ssYVaArbv35RaoKgN1F8/09oDUogUBAkpSdnS1JamhoUF9fn0pLS60+kydPVmFhoerr6yVJ9fX1mjZtmhVOJKmsrEzBYFCHDh065/v09PQoGAxGbAASy8EaFABxFHNACYfDuvfee3XNNddo6tSpkiS/3y+Xy6WsrKyIvrm5ufL7/VafvwwnZ9vPtp1LbW2tPB6PtRUUFMRaNoDB4HDImZKS6CoADGExB5TKykodPHhQGzZsGMx6zqmmpkaBQMDajh8/Hvf3BHA+XGYMIL5imqOtqqrS5s2btXv3bo0dO9ba7/V61dvbq46OjohZlLa2Nnm9XqvPu+++G/F6Z6/yOdvni9LT05Wenh5LqQDi5EJv1GaMkcPhiHM1AIaaqGZQjDGqqqrSpk2btHPnTo0fPz6ifcaMGUpLS9OOHTusfc3NzWppaZHP55Mk+Xw+NTU1qb293eqzfft2ud1uFRUVDeRYAFwkFxo4jIxMOBTnagAMRVHNoFRWVmr9+vV67bXXNHLkSGvNiMfj0fDhw+XxeLR06VJVV1crOztbbrdb99xzj3w+n2bNmiVJmjt3roqKirR48WI9/vjj8vv9WrlypSorK5klAYYa8/8CCpckA4hSVJ8aa9askSTNnj07Yv/zzz+vO++8U5L05JNPyul0qry8XD09PSorK9Nzzz1n9U1JSdHmzZu1fPly+Xw+jRgxQhUVFXrkkUcGdiQAbMcYIxPul8R/PgBEZ0D3QUkU7oMCJF7Hnw/oo62/Om+fYVl5mnxTtdIyPBepKgB2dtHugwIA52cUDrEGBUD0CCgA4saY8P87xQMA0SGgAIgfaw0KAESHgAIgfgyneADEhoACIG6MMTIhZlAARI+AAiCOOMUDIDYEFABxYwx3kgUQGwIKgPhhkSyAGBFQAMTENXKURoy59Lx9+j4PKnD88MUpCMCQQkABEBNniksprozzdzJhhXo/vzgFARhSCCgAYuJwOuRISUl0GQCGKAIKgNg4nHI4CCgA4oOAAiAmDoeTGRQAcUNAARATh8Mph5OAAiA+CCgAYkNAARBHBBQAMXE4HXI4+AgBEB98ugCIDWtQAMQRAQVATBxcxQMgjggoAGLicHAfFADxQ0ABEBuHUw5naqKrADBEEVAAxIRTPADiiYACIDYOhxzOC/kIMTLGxL0cAEMLAQVATBwOxwX1M+GwjAnHuRoAQw0BBUBcGROWwgQUANEhoACIK2PCMiaU6DIAJBkCCoC4MuGQDDMoAKJEQAEQX+GwTJgZFADRIaAAiKszp3iYQQEQHQIKgLgy4TCneABEjYACIL5M6MwGAFEgoACIK2ZQAMQiqoBSW1urq666SiNHjlROTo4WLlyo5ubmiD6zZ88+8yNif7HdfffdEX1aWlq0YMECZWRkKCcnRw888ID6+/sHfjQAbMcYFskCiF5Uv/RVV1enyspKXXXVVerv79fPfvYzzZ07V4cPH9aIESOsfnfddZceeeQR63FGRob1dygU0oIFC+T1evXOO++otbVVS5YsUVpamh599NFBOCQAdsKdZAHEIqqAsnXr1ojH69atU05OjhoaGnTddddZ+zMyMuT1es/5Gr/73e90+PBhvfnmm8rNzdWVV16pX/ziF1qxYoUefvhhuVyuGA4DgF0ZE2IGBUDUBrQGJRAISJKys7Mj9r/44osaPXq0pk6dqpqaGn322WdWW319vaZNm6bc3FxrX1lZmYLBoA4dOnTO9+np6VEwGIzYACRe6rARcqae/z8VoZ7PFOr57Lx9AOCLoppB+UvhcFj33nuvrrnmGk2dOtXa/8Mf/lDjxo1Tfn6+Dhw4oBUrVqi5uVmvvPKKJMnv90eEE0nWY7/ff873qq2t1erVq2MtFUCcjBg9Tq7MUeruaP3KPj3Bk+oOtMl9yeSLWBmAZBdzQKmsrNTBgwf19ttvR+xftmyZ9fe0adOUl5enOXPm6OjRo7r88stjeq+amhpVV1dbj4PBoAoKCmIrHMDgcaZc8K8aA0A0YjrFU1VVpc2bN+utt97S2LFjz9u3pKREknTkyBFJktfrVVtbW0Sfs4+/at1Kenq63G53xAYg8RzOFMnB3QoADL6oPlmMMaqqqtKmTZu0c+dOjR8//muf09jYKEnKy8uTJPl8PjU1Nam9vd3qs337drndbhUVFUVTDoAEczhT5HASUAAMvqhO8VRWVmr9+vV67bXXNHLkSGvNiMfj0fDhw3X06FGtX79eN9xwg0aNGqUDBw7ovvvu03XXXafi4mJJ0ty5c1VUVKTFixfr8ccfl9/v18qVK1VZWan09PTBP0IAccMMCoB4ieqTZc2aNQoEApo9e7by8vKs7aWXXpIkuVwuvfnmm5o7d64mT56s+++/X+Xl5Xr99det10hJSdHmzZuVkpIin8+nv/u7v9OSJUsi7psCIDk4nU7WoACIi6hmUIwx520vKChQXV3d177OuHHj9MYbb0Tz1gDsyJkiBzMoAOKATxYAMeMUD4B44ZMFQMycLJIFECd8sgCIHTMoAOKETxYAMXNyozYAcUJAARA7FskCiBM+WQDEzOFwShcwg2KM+dqrAAHgLxFQAMTsgk/vhEOSCCgALhwBBUDchUP9zKAAiAoBBUDcmXBIIqAAiAIBBUDcmXC/ZMKJLgNAEiGgAIg7TvEAiBYBBUDccYoHQLQIKADizoRDzKAAiAoBBUDcmXC/JNagALhwBBQAcWdCnOIBEB0CCoC4C4dZJAsgOgQUAHFnQlxmDCA6BBQAccciWQDRIqAAGJCR+ZPkTHWdt09X+zH1d3depIoADAWpiS4AQGKFw2GFw7GffkkbMUoOZ8p5+/R9FlBfb7f6+/tjfh+Hw6GUlPO/D4Chg4ACfMO99NJLWrJkSczPn1V0iVZVfE+eEenn7Xfdtdfq8J8/ifl9pk2bpvfffz/m5wNILgQU4BvOGDOgmY3u7r4LWl8SCoUG9D6hUCjm5wJIPgQUAAPS2///F8B2hzJ0sq9A3aERSnH0y5PWrlFp/gRXCCAZEVAADMiZgHImnOw/fb06Qx71m3Q5FdIwZ6cKh3+g8cObEl0mgCRDQAEwIH39IfWbVL3T8QP1mBHW/rBS9Vk4S3/qmqk0R4+MHAmsEkCy4TJjAAPS1x/S7//3VvWYjHO2h5Wmps7vKdA/5iJXBiCZEVAADEhvf0hhI+m8MyTMngCIDgEFwID09of5HUAAg46AAmBA+vq4jT2AwUdAATAgvf0h+bI2KcXRe852h8KaMuIduVNjv0kbgG+eqALKmjVrVFxcLLfbLbfbLZ/Ppy1btljt3d3dqqys1KhRo5SZmany8nK1tbVFvEZLS4sWLFigjIwM5eTk6IEHHhjQzZsAJFZfKCyX43Ndm/WyRqT8r1LUK8nIoZDSnV2akPG+xg07KKf4NWMAFy6qy4zHjh2rxx57TBMnTpQxRi+88IJuvvlm7d+/X1dccYXuu+8+/fa3v9XGjRvl8XhUVVWlW265RX/4wx8knbkT5IIFC+T1evXOO++otbVVS5YsUVpamh599NG4HCCA+AqHjbbtOyLPiI/1eehDtfZcps/DbqU4+jQq7YSCruM6KOl/T3cnulQAScRhBnjyODs7W0888YRuvfVWjRkzRuvXr9ett94qSfrwww81ZcoU1dfXa9asWdqyZYtuvPFGnThxQrm5uZKktWvXasWKFTp58qRcrvP/IupZwWBQHo9Hd9555wU/B8C5HTlyRDt37kx0GV8rOzvb+mwBkJx6e3u1bt06BQIBud3u8/aN+UZtoVBIGzduVFdXl3w+nxoaGtTX16fS0lKrz+TJk1VYWGgFlPr6ek2bNs0KJ5JUVlam5cuX69ChQ/rOd75zzvfq6elRT0+P9TgYDEqSFi9erMzMzFgPAYCkbdu2JU1AWbp0aaLLADAAnZ2dWrdu3QX1jTqgNDU1yefzqbu7W5mZmdq0aZOKiorU2Ngol8ulrKysiP65ubny+8/8Foff748IJ2fbz7Z9ldraWq1evfpL+2fOnPm1CQzA+R05ciTRJVyQjIwMXX311YkuA8AAnJ1guBBRX8UzadIkNTY2au/evVq+fLkqKip0+PDhaF8mKjU1NQoEAtZ2/PjxuL4fAABIrKhnUFwulyZMmCBJmjFjhvbt26enn35at912m3p7e9XR0RExi9LW1iav1ytJ8nq9evfddyNe7+xVPmf7nEt6errS09OjLRUAACSpAd8HJRwOq6enRzNmzFBaWpp27NhhtTU3N6ulpUU+n0+S5PP51NTUpPb2dqvP9u3b5Xa7VVRUNNBSAADAEBHVDEpNTY3mz5+vwsJCnT59WuvXr9euXbu0bds2eTweLV26VNXV1crOzpbb7dY999wjn8+nWbNmSZLmzp2roqIiLV68WI8//rj8fr9WrlypyspKZkgAAIAlqoDS3t6uJUuWqLW1VR6PR8XFxdq2bZuuv/56SdKTTz4pp9Op8vJy9fT0qKysTM8995z1/JSUFG3evFnLly+Xz+fTiBEjVFFRoUceeWRwjwoAACS1Ad8HJRHO3gflQq6jBnB+69ev16JFixJdxtcqLi7WH//4x0SXAWAAovn+5rd4AACA7RBQAACA7RBQAACA7RBQAACA7cT8WzwAhoaxY8dq4cKFiS7ja1166aWJLgHARcRVPAAA4KLgKh4AAJDUCCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2CCgAAMB2ogooa9asUXFxsdxut9xut3w+n7Zs2WK1z549Ww6HI2K7++67I16jpaVFCxYsUEZGhnJycvTAAw+ov79/cI4GAAAMCanRdB47dqwee+wxTZw4UcYYvfDCC7r55pu1f/9+XXHFFZKku+66S4888oj1nIyMDOvvUCikBQsWyOv16p133lFra6uWLFmitLQ0Pfroo4N0SAAAINk5jDFmIC+QnZ2tJ554QkuXLtXs2bN15ZVX6qmnnjpn3y1btujGG2/UiRMnlJubK0lau3atVqxYoZMnT8rlcl3QewaDQXk8HgUCAbnd7oGUDwAALpJovr9jXoMSCoW0YcMGdXV1yefzWftffPFFjR49WlOnTlVNTY0+++wzq62+vl7Tpk2zwokklZWVKRgM6tChQ1/5Xj09PQoGgxEbAAAYuqI6xSNJTU1N8vl86u7uVmZmpjZt2qSioiJJ0g9/+EONGzdO+fn5OnDggFasWKHm5ma98sorkiS/3x8RTiRZj/1+/1e+Z21trVavXh1tqQAAIElFHVAmTZqkxsZGBQIBvfzyy6qoqFBdXZ2Kioq0bNkyq9+0adOUl5enOXPm6OjRo7r88stjLrKmpkbV1dXW42AwqIKCgphfDwAA2FvUp3hcLpcmTJigGTNmqLa2VtOnT9fTTz99zr4lJSWSpCNHjkiSvF6v2traIvqcfez1er/yPdPT060rh85uAABg6BrwfVDC4bB6enrO2dbY2ChJysvLkyT5fD41NTWpvb3d6rN9+3a53W7rNBEAAEBUp3hqamo0f/58FRYW6vTp01q/fr127dqlbdu26ejRo1q/fr1uuOEGjRo1SgcOHNB9992n6667TsXFxZKkuXPnqqioSIsXL9bjjz8uv9+vlStXqrKyUunp6XE5QAAAkHyiCijt7e1asmSJWltb5fF4VFxcrG3btun666/X8ePH9eabb+qpp55SV1eXCgoKVF5erpUrV1rPT0lJ0ebNm7V8+XL5fD6NGDFCFRUVEfdNAQAAGPB9UBKB+6AAAJB8Lsp9UAAAAOKFgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGyHgAIAAGwnNdEFxMIYI0kKBoMJrgQAAFyos9/bZ7/HzycpA8rp06clSQUFBQmuBAAAROv06dPyeDzn7eMwFxJjbCYcDqu5uVlFRUU6fvy43G53oktKWsFgUAUFBYzjIGAsBw9jOTgYx8HDWA4OY4xOnz6t/Px8OZ3nX2WSlDMoTqdTl1xyiSTJ7Xbzj2UQMI6Dh7EcPIzl4GAcBw9jOXBfN3NyFotkAQCA7RBQAACA7SRtQElPT9eqVauUnp6e6FKSGuM4eBjLwcNYDg7GcfAwlhdfUi6SBQAAQ1vSzqAAAIChi4ACAABsh4ACAABsh4ACAABsJykDyrPPPqtLL71Uw4YNU0lJid59991El2Q7u3fv1k033aT8/Hw5HA69+uqrEe3GGD300EPKy8vT8OHDVVpaqo8++iiiz6lTp7Ro0SK53W5lZWVp6dKl6uzsvIhHkXi1tbW66qqrNHLkSOXk5GjhwoVqbm6O6NPd3a3KykqNGjVKmZmZKi8vV1tbW0SflpYWLViwQBkZGcrJydEDDzyg/v7+i3koCbVmzRoVFxdbN7ny+XzasmWL1c4Yxu6xxx6Tw+HQvffea+1jPC/Mww8/LIfDEbFNnjzZamccE8wkmQ0bNhiXy2X+4z/+wxw6dMjcddddJisry7S1tSW6NFt54403zD/+4z+aV155xUgymzZtimh/7LHHjMfjMa+++qr54x//aP72b//WjB8/3nz++edWn3nz5pnp06ebPXv2mN///vdmwoQJ5o477rjIR5JYZWVl5vnnnzcHDx40jY2N5oYbbjCFhYWms7PT6nP33XebgoICs2PHDvPee++ZWbNmmb/+67+22vv7+83UqVNNaWmp2b9/v3njjTfM6NGjTU1NTSIOKSH+67/+y/z2t781f/rTn0xzc7P52c9+ZtLS0szBgweNMYxhrN59911z6aWXmuLiYvPjH//Y2s94XphVq1aZK664wrS2tlrbyZMnrXbGMbGSLqBcffXVprKy0nocCoVMfn6+qa2tTWBV9vbFgBIOh43X6zVPPPGEta+jo8Okp6ebX//618YYYw4fPmwkmX379ll9tmzZYhwOh/mf//mfi1a73bS3txtJpq6uzhhzZtzS0tLMxo0brT4ffPCBkWTq6+uNMWfCotPpNH6/3+qzZs0a43a7TU9Pz8U9ABv51re+Zf7t3/6NMYzR6dOnzcSJE8327dvN9773PSugMJ4XbtWqVWb69OnnbGMcEy+pTvH09vaqoaFBpaWl1j6n06nS0lLV19cnsLLkcuzYMfn9/ohx9Hg8Kikpscaxvr5eWVlZmjlzptWntLRUTqdTe/fuveg120UgEJAkZWdnS5IaGhrU19cXMZaTJ09WYWFhxFhOmzZNubm5Vp+ysjIFg0EdOnToIlZvD6FQSBs2bFBXV5d8Ph9jGKPKykotWLAgYtwk/k1G66OPPlJ+fr4uu+wyLVq0SC0tLZIYRztIqh8L/OSTTxQKhSL+MUhSbm6uPvzwwwRVlXz8fr8knXMcz7b5/X7l5OREtKempio7O9vq800TDod177336pprrtHUqVMlnRknl8ulrKysiL5fHMtzjfXZtm+KpqYm+Xw+dXd3KzMzU5s2bVJRUZEaGxsZwyht2LBB77//vvbt2/elNv5NXriSkhKtW7dOkyZNUmtrq1avXq1rr71WBw8eZBxtIKkCCpBIlZWVOnjwoN5+++1El5KUJk2apMbGRgUCAb388suqqKhQXV1dostKOsePH9ePf/xjbd++XcOGDUt0OUlt/vz51t/FxcUqKSnRuHHj9Jvf/EbDhw9PYGWQkuwqntGjRyslJeVLq6jb2trk9XoTVFXyOTtW5xtHr9er9vb2iPb+/n6dOnXqGznWVVVV2rx5s9566y2NHTvW2u/1etXb26uOjo6I/l8cy3ON9dm2bwqXy6UJEyZoxowZqq2t1fTp0/X0008zhlFqaGhQe3u7/uqv/kqpqalKTU1VXV2dnnnmGaWmpio3N5fxjFFWVpa+/e1v68iRI/y7tIGkCigul0szZszQjh07rH3hcFg7duyQz+dLYGXJZfz48fJ6vRHjGAwGtXfvXmscfT6fOjo61NDQYPXZuXOnwuGwSkpKLnrNiWKMUVVVlTZt2qSdO3dq/PjxEe0zZsxQWlpaxFg2NzerpaUlYiybmpoiAt/27dvldrtVVFR0cQ7EhsLhsHp6ehjDKM2ZM0dNTU1qbGy0tpkzZ2rRokXW34xnbDo7O3X06FHl5eXx79IOEr1KN1obNmww6enpZt26debw4cNm2bJlJisrK2IVNc6s8N+/f7/Zv3+/kWT+9V//1ezfv9/8+c9/Nsacucw4KyvLvPbaa+bAgQPm5ptvPudlxt/5znfM3r17zdtvv20mTpz4jbvMePny5cbj8Zhdu3ZFXIr42WefWX3uvvtuU1hYaHbu3Gnee+894/P5jM/ns9rPXoo4d+5c09jYaLZu3WrGjBnzjboU8cEHHzR1dXXm2LFj5sCBA+bBBx80DofD/O53vzPGMIYD9ZdX8RjDeF6o+++/3+zatcscO3bM/OEPfzClpaVm9OjRpr293RjDOCZa0gUUY4z55S9/aQoLC43L5TJXX3212bNnT6JLsp233nrLSPrSVlFRYYw5c6nxz3/+c5Obm2vS09PNnDlzTHNzc8RrfPrpp+aOO+4wmZmZxu12mx/96Efm9OnTCTiaxDnXGEoyzz//vNXn888/N//wD/9gvvWtb5mMjAzzgx/8wLS2tka8zn//93+b+fPnm+HDh5vRo0eb+++/3/T19V3ko0mcv//7vzfjxo0zLpfLjBkzxsyZM8cKJ8YwhgP1xYDCeF6Y2267zeTl5RmXy2UuueQSc9ttt5kjR45Y7YxjYjmMMSYxczcAAADnllRrUAAAwDcDAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANgOAQUAANjO/wWa8zuk5DyzkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "student.test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
